<!DOCTYPE HTML>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" /> 
    <title> - 天地维杰网</title>
    <meta name="keywords" content="系统架构,shutdown,不与天斗,Domino,博客,程序员,架构师,笔记,技术,分享,java,Redis">
    
    <meta property="og:title" content="">
    <meta property="og:site_name" content="天地维杰网">
    <meta property="og:image" content="/img/author.jpg"> 
    <meta name="title" content=" - 天地维杰网" />
    <meta name="description" content="天地维杰网 | 博客 | 软件 | 架构 | Java "> 
    <link rel="shortcut icon" href="http://www.shutdown.cn/img/favicon.ico" />
    <link rel="apple-touch-icon" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link rel="apple-touch-icon-precomposed" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link href="http://www.shutdown.cn/js/vendor/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/main.css" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/syntax.css" rel="stylesheet" type="text/css" />
    <script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
     fancybox: true, 
    motion: true
  };
</script>
<script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-7826003325059020" crossorigin="anonymous"></script>
</head>
<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">
<div class="container one-collumn sidebar-position-left page-home  ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"> <div class="site-meta  custom-logo ">

  <div class="custom-logo-site-title">
    <a href="http://www.shutdown.cn"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">天地维杰网</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle">人如秋鸿来有信，事若春梦了无痕</p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
    <ul id="menu" class="menu">
      
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />首页
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/redis/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-battery-full"></i> <br />Redis
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/java/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-coffee"></i> <br />java
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/linux/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-linux"></i> <br />linux
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/daily/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bug"></i> <br />日常问题
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/spring/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-child"></i> <br />Spring和Springboot
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/mac/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-fire"></i> <br />Mac相关
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/middleware/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-gavel"></i> <br />中间件
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jiagou/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-rocket"></i> <br />架构
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/python/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-ship"></i> <br />python
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/front/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bolt"></i> <br />前端
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jvm/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-balance-scale"></i> <br />jvm
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/c/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-battery-empty"></i> <br />c语言
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/web3/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-web3"></i> <br />web3
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/post/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />归档
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/about/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />关于
          </a>
        </li>
      
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger"> <i class="menu-item-icon fa fa-search fa-fw"></i> <br /> 搜索</a>
      </li>
    </ul>
    <div class="site-search">
      <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>

    </div>
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
<section id="posts" class="posts-expand">
  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
    <header class="post-header">
      <h1 class="post-title" itemprop="name headline">
        <a class="post-title-link" href="http://www.shutdown.cn/post/zookeeper%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%94%99%E8%AF%AFpacket-len-is-out-of-range/" itemprop="url">
        
        </a>
      </h1>
      <div class="post-meta">
      <span class="post-time">
<span class="post-meta-item-icon">
    <i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">时间：</span>
<time itemprop="dateCreated" datetime="2016-03-22T13:04:35+08:00" content="0001-01-01">
    0001-01-01
</time>
</span> 
      
      
       <span>
&nbsp; | &nbsp;
<span class="post-meta-item-icon">
    <i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读：</span>
<span class="leancloud-visitors-count">248 字 ~2分钟</span>
</span>
      </div>
    </header>
    <div class="post-body" itemprop="articleBody">
    

    

<p>这是一个生产环境使用zookeeper异常的情况，错误是<code>java.io.IOException: Packet len8854970 is out of range!</code>。然后就换了一个namespace，就没有在出错，以为是偶然发生，所以没有重视。但是年后居然又出现问题，才意识到严重性。分析之后发现，每隔一段时间，某一个znode节点下超过客户端所设置的大小，客户端连接会失败，zkCli.sh操作该节点也会失败。如果对于简单依赖zookeeper的系统，这种错误可以容忍（但是必须解决）；如果是强依赖zookeeper的系统，这种错误可以说是灾难。</p>

<h1 id="1-发现问题">1 发现问题</h1>

<p>问题的发现比较曲折，首先是发现服务器磁盘写满了（吐槽下运维居然没有对磁盘添加监控），致使项目中的报警功能失效。然后就把无用日志删除，习（tou）惯（lan）的把异常的应用重启了下。幸好有个比较好的习惯就是，项目启动成功后，都会打看日志跟踪下，确定无误才会关掉终端。结果就被错误日志刷屏了：</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java" data-lang="java"><span style="color:#000">ERROR</span> <span style="color:#000">2017</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">02</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">20</span> <span style="color:#000">10</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">45</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">44</span><span style="color:#ce5c00;font-weight:bold">,</span><span style="color:#000">729</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">Curator</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">Framework</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">0</span><span style="color:#ce5c00;font-weight:bold">]</span> <span style="color:#000">o</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">a</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">c</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">f</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">i</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">logError</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">557</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">-</span> <span style="color:#000">Background</span> <span style="color:#000">retry</span> <span style="color:#000">gave</span> <span style="color:#000">up</span>
<span style="color:#000">org</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">apache</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">curator</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorConnectionLossException</span><span style="color:#ce5c00;font-weight:bold">:</span> <span style="color:#000">KeeperErrorCode</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">ConnectionLoss</span>
        <span style="color:#000">at</span> <span style="color:#000">org</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">apache</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">curator</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">framework</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">imps</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">performBackgroundOperation</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">838</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">curator</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">framework</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">2</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">10</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">jar</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">org</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">apache</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">curator</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">framework</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">imps</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">backgroundOperationsLoop</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">809</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">curator</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">framework</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">2</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">10</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">jar</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">org</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">apache</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">curator</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">framework</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">imps</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">access$300</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">64</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">curator</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">framework</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">2</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">10</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">jar</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">org</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">apache</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">curator</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">framework</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">imps</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">CuratorFrameworkImpl$4</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">call</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">CuratorFrameworkImpl</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">267</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">curator</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">framework</span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#000">2</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">10</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">jar</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">util</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">concurrent</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">FutureTask</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">run</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">FutureTask</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">266</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">util</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">concurrent</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">ScheduledThreadPoolExecutor$ScheduledFutureTask</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">access$201</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">ScheduledThreadPoolExecutor</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">180</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">util</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">concurrent</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">ScheduledThreadPoolExecutor$ScheduledFutureTask</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">run</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">ScheduledThreadPoolExecutor</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">293</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">util</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">concurrent</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">ThreadPoolExecutor</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">runWorker</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">ThreadPoolExecutor</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1142</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">util</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">concurrent</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">ThreadPoolExecutor$Worker</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">run</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">ThreadPoolExecutor</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">617</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span>
        <span style="color:#000">at</span> <span style="color:#000">java</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">lang</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">Thread</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">run</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">Thread</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">java</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">745</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">[</span><span style="color:#000">na</span><span style="color:#ce5c00;font-weight:bold">:</span><span style="color:#000">1</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">8</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">0_111</span><span style="color:#ce5c00;font-weight:bold">]</span></code></pre></div>
<p>因为zookeeper的客户端使用的是<a href="http://curator.apache.org/">Apache Curator</a>，与zookeeper的连接断开会重新创建连接，所以会出现大量的连接失败异常。</p>

<h1 id="2-分析问题">2 分析问题</h1>

<p>在这个项目中，对于zookeepr的使用有下面几种场景：</p>

<ol>
<li>服务状态数据：将服务状态写入znode节点</li>
<li>服务注册：将服务地址写入znode节点</li>
<li>服务发现：获取znode节点中的可用服务地址</li>
<li>监听节点：监听某个znode节点或该节点的字节点状态</li>
</ol>

<p>这几种情况都需要与zookeeper的znode节点创建连接，然后执行操作。根据错误提示，<code>java.io.IOException: Packet len8854970 is out of range!</code>，out of range就是超过了某个限制，只能查看代码了。</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java" data-lang="java"><span style="color:#204a87;font-weight:bold">protected</span> <span style="color:#204a87;font-weight:bold">final</span> <span style="color:#000">ByteBuffer</span> <span style="color:#000">lenBuffer</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">ByteBuffer</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">allocateDirect</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">4</span><span style="color:#ce5c00;font-weight:bold">)</span><span style="color:#ce5c00;font-weight:bold">;</span>
<span style="color:#204a87;font-weight:bold">protected</span> <span style="color:#000">ByteBuffer</span> <span style="color:#000">incomingBuffer</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">lenBuffer</span><span style="color:#ce5c00;font-weight:bold">;</span>

<span style="color:#204a87;font-weight:bold">protected</span> <span style="color:#204a87;font-weight:bold">void</span> <span style="color:#000">readLength</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#204a87;font-weight:bold">throws</span> <span style="color:#000">IOException</span> <span style="color:#ce5c00;font-weight:bold">{</span>
    <span style="color:#204a87;font-weight:bold">int</span> <span style="color:#000">len</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">incomingBuffer</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">getInt</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#ce5c00;font-weight:bold">)</span><span style="color:#ce5c00;font-weight:bold">;</span>
    <span style="color:#204a87;font-weight:bold">if</span> <span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">len</span> <span style="color:#ce5c00;font-weight:bold">&lt;</span> <span style="color:#000">0</span> <span style="color:#ce5c00;font-weight:bold">|</span><span style="color:#ce5c00;font-weight:bold">|</span> <span style="color:#000">len</span> <span style="color:#ce5c00;font-weight:bold">&gt;</span><span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">ClientCnxn</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">packetLen</span><span style="color:#ce5c00;font-weight:bold">)</span> <span style="color:#ce5c00;font-weight:bold">{</span>
        <span style="color:#204a87;font-weight:bold">throw</span> <span style="color:#204a87;font-weight:bold">new</span> <span style="color:#000">IOException</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#4e9a06">&#34;Packet len&#34;</span> <span style="color:#ce5c00;font-weight:bold">+</span> <span style="color:#000">len</span> <span style="color:#ce5c00;font-weight:bold">+</span> <span style="color:#4e9a06">&#34; is out of range!&#34;</span><span style="color:#ce5c00;font-weight:bold">)</span><span style="color:#ce5c00;font-weight:bold">;</span>
    <span style="color:#ce5c00;font-weight:bold">}</span>
    <span style="color:#000">incomingBuffer</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">ByteBuffer</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">allocate</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#000">len</span><span style="color:#ce5c00;font-weight:bold">)</span><span style="color:#ce5c00;font-weight:bold">;</span>
<span style="color:#ce5c00;font-weight:bold">}</span>

<span style="color:#204a87;font-weight:bold">public</span> <span style="color:#204a87;font-weight:bold">static</span> <span style="color:#204a87;font-weight:bold">final</span> <span style="color:#204a87;font-weight:bold">int</span> <span style="color:#000">packetLen</span> <span style="color:#ce5c00;font-weight:bold">=</span> <span style="color:#000">Integer</span><span style="color:#ce5c00;font-weight:bold">.</span><span style="color:#c4a000">getInteger</span><span style="color:#ce5c00;font-weight:bold">(</span><span style="color:#4e9a06">&#34;jute.maxbuffer&#34;</span><span style="color:#ce5c00;font-weight:bold">,</span> <span style="color:#000">4096</span> <span style="color:#ce5c00;font-weight:bold">*</span> <span style="color:#000">1024</span><span style="color:#ce5c00;font-weight:bold">)</span><span style="color:#ce5c00;font-weight:bold">;</span></code></pre></div>
<p>从代码就能够很容易的看出，这个错误是因为<code>len</code>小于0或大于<code>packetLen</code>，根据代码逻辑，<code>len</code>不小于0，那就是大于<code>packetLen</code>。而<code>packetLen</code>的值是<code>jute.maxbuffer</code>系统变量定义或默认的4096 * 1024（4M）。</p>

<p>继续深扒代码，因为代码比较长，这里就不写了。大体逻辑就是，创建与zookeeper连接之后，要对某个节点进行读写操作，为了提高吞吐量，先判断下该节点数据量大小是否超过设置的<code>jute.maxbuffer</code>，如果是，就抛出异常。在zookeeper客户端中，这一部分异常的处理比较粗糙，因为注释上也写着“this is ugly, you have a better way speak up”。</p>

<h1 id="3-解决问题">3 解决问题</h1>

<p>根据上面的纠错，答案就很明显了。只有两种方案：</p>

<ul>
<li>把待操作节点的大小减下来，小于默认的4M</li>
<li>把默认的<code>jute.maxbuffer</code>大小提高</li>
</ul>

<p>对于第一种方式，需要根据自身具体情况具体操作。这里没有什么有效建议。</p>

<p>对于第二种方式，就比较简单了。只要在创建Zookeeper对象之前，设置<code>System.setProperty(&quot;jute.maxbuffer&quot;, 4096 * 1024 * 10 + &quot;&quot;);</code>，这里的大小根据自己的系统设置，我这里只是一个测试值（如果设置太大，这个节点真的比较大的话，会影响吞吐）。</p>

<blockquote>
<p>因为我这里使用的是<a href="http://curator.apache.org/">Apache Curator</a>，不需要自己创建Zookeeper对象，所以需要在创建CuratorFramework对象之前添加这个变量。</p>
</blockquote>

<p>解决问题要彻底，不能留下祸患。（此处应该伴随阴笑和冷风。。。）</p>

<p>java客户端的问题解决了，但是通过zkCli.sh连接时，还是会出现这个问题。报错如下：</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-fallback" data-lang="fallback">2017-02-20 12:08:03,999 [myid:] - WARN  [main-SendThread(localhost:2181):ClientCnxn$SendThread@1102] - Session 0x1591b713cefd2b3 for server localhost/127.0.0.1:2181, unexpected error, closing socket connection and attempting reconnect
java.io.IOException: Packet len8854970 is out of range!
	at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)

WATCHER::

WatchedEvent state:Disconnected type:None path:null
Exception in thread &#34;main&#34; org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /it-monitor/com/wfj/monitor/sales-overview-info
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1500)
	at org.apache.zookeeper.ZKUtil.listSubTreeBFS(ZKUtil.java:114)
	at org.apache.zookeeper.ZKUtil.deleteRecursive(ZKUtil.java:49)
	at org.apache.zookeeper.ZooKeeperMain.processZKCmd(ZooKeeperMain.java:703)
	at org.apache.zookeeper.ZooKeeperMain.processCmd(ZooKeeperMain.java:588)
	at org.apache.zookeeper.ZooKeeperMain.executeLine(ZooKeeperMain.java:360)
	at org.apache.zookeeper.ZooKeeperMain.run(ZooKeeperMain.java:323)
	at org.apache.zookeeper.ZooKeeperMain.main(ZooKeeperMain.java:282)</code></pre></div>
<p>找到zkCli.sh，最下面的java命令中添加对<code>jute.maxbuffer</code>的定义（使用D参数）：</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-sh" data-lang="sh"><span style="color:#4e9a06">&#34;</span><span style="color:#000">$JAVA</span><span style="color:#4e9a06">&#34;</span> <span style="color:#4e9a06">&#34;</span><span style="color:#4e9a06">-Dzookeeper.log.dir=</span><span style="color:#4e9a06">${</span><span style="color:#000">ZOO_LOG_DIR</span><span style="color:#4e9a06">}</span><span style="color:#4e9a06">&#34;</span> <span style="color:#4e9a06">&#34;</span><span style="color:#4e9a06">-Dzookeeper.root.logger=</span><span style="color:#4e9a06">${</span><span style="color:#000">ZOO_LOG4J_PROP</span><span style="color:#4e9a06">}</span><span style="color:#4e9a06">&#34;</span> <span style="color:#4e9a06">\
</span><span style="color:#4e9a06"></span>     <span style="color:#4e9a06">&#34;-Djute.maxbuffer=41943040&#34;</span> <span style="color:#4e9a06">\
</span><span style="color:#4e9a06"></span>     -cp <span style="color:#4e9a06">&#34;</span><span style="color:#000">$CLASSPATH</span><span style="color:#4e9a06">&#34;</span> <span style="color:#000">$CLIENT_JVMFLAGS</span> <span style="color:#000">$JVMFLAGS</span> <span style="color:#4e9a06">\
</span><span style="color:#4e9a06"></span>     org.apache.zookeeper.ZooKeeperMain <span style="color:#4e9a06">&#34;</span><span style="color:#000">$@</span><span style="color:#4e9a06">&#34;</span></code></pre></div>
<p>当然，为了运维方便，可以把<code>jute.maxbuffer</code>的值设置成变量，通过修改配置来设置值。避免因为修改sh脚本出现其他问题。</p>

    </div>
    <footer class="post-footer">
     

     <div class="post-nav">
    <div class="post-nav-next post-nav-item">
    
        <a href="http://www.shutdown.cn/post/xen%E8%99%9A%E6%8B%9F%E5%8C%96%E6%8A%80%E6%9C%AF%E4%B8%ADpv%E5%92%8Chvm%E7%9A%84%E5%8C%BA%E5%88%AB/" rel="next" title="">
        <i class="fa fa-chevron-left"></i> 
        </a>
    
    </div>

    <div class="post-nav-prev post-nav-item">
    
        <a href="http://www.shutdown.cn/post/zookeeper%E9%A2%91%E7%B9%81%E6%8A%A5connection-loss/" rel="prev" title="">
         <i class="fa fa-chevron-right"></i>
        </a>
    
    </div>
</div>
      
     
     
     






    </footer>
  </article>
</section>

          </div>
        </div>
        <div class="sidebar-toggle">
  <div class="sidebar-toggle-line-wrap">
    <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
  </div>
</div>
<aside id="sidebar" class="sidebar">
  <div class="sidebar-inner">

    <section class="site-overview sidebar-panel  sidebar-panel-active ">
      <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
        src="http://www.shutdown.cn/img/author.jpg"
        alt="不与天斗Domino" />
    <p class="site-author-name" itemprop="name">不与天斗Domino</p>
    <p class="site-description motion-element" itemprop="description"> 
        Programmer &amp; Architect</p>
</div>
      <nav class="site-state motion-element">
    <div class="site-state-item site-state-posts">
      <a href="http://www.shutdown.cn/post/">
        <span class="site-state-item-count">183</span>
        <span class="site-state-item-name">日志</span>
      </a>
    </div>
    <div class="site-state-item site-state-categories">    
        <a href="http://www.shutdown.cn/categories/">      
         
        <span class="site-state-item-count">15</span>
        
        <span class="site-state-item-name">分类</span>
        
        </a>
    </div>

    <div class="site-state-item site-state-tags">
        <a href="http://www.shutdown.cn/tags/">
         
        <span class="site-state-item-count">224</span>
        
        <span class="site-state-item-name">标签</span>
        </a>
    </div>
</nav>
      
      

      

      <div class="links-of-blogroll motion-element inline">
<script type="text/javascript" src="//rf.revolvermaps.com/0/0/8.js?i=&amp;m=0&amp;s=220&amp;c=ff0000&amp;cr1=ffffff&amp;f=arial&amp;l=33&amp;bv=35" async="async"></script>
</div>

    </section>
    
  </div>
</aside>

      </div>
    </main>
   
    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  <span itemprop="copyrightYear">  &copy; 
  2013 - 2023</span>
  <span class="with-love"><i class="fa fa-heart"></i></span>
  <span class="author" itemprop="copyrightHolder">天地维杰网</span>
  <span class="icp" itemprop="copyrightHolder"><a href="https://beian.miit.gov.cn/" target="_blank">京ICP备13019191号-1</a></span>
</div>
<div class="powered-by">
  Powered by - <a class="theme-link" href="http://gohugo.io" target="_blank" title="hugo" >Hugo v0.63.2</a>
</div>
<div class="theme-info">
  Theme by - <a class="theme-link" href="https://github.com/xtfly/hugo-theme-next" target="_blank"> NexT
  </a>
</div>


      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span id="scrollpercent"><span>0</span>%</span>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery/index.js?v=2.1.3"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/fastclick/lib/fastclick.min.js?v=1.0.6"></script> 
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.min.js?v=1.2.1"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.ui.min.js?v=1.2.1"></script>
<script src="http://www.shutdown.cn/js/vendor/ua-parser-js/dist/ua-parser.min.js?v=0.7.9"></script>

<script src="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.pack.js?v=2.1.5"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/utils.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/motion.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/affix.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/schemes/pisces.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/scrollspy.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/post-details.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/toc.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/bootstrap.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/search.js"></script>
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({
    extensions: ["tex2jax.js"],
    jax: ["input/TeX", "output/HTML-CSS"],
    tex2jax: {
      inlineMath: [ ['$','$'] ],
      displayMath: [ ['$$','$$'] ],
      processEscapes: true
    },
    "HTML-CSS": { fonts: ["TeX"] }
  });
</script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_HTMLorMML' async></script>
</body>
</html>